﻿using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Diagnostics.Eventing.Reader;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Dapper;
using DotnetSpider.DataFlow;
using DotnetSpider.DataFlow.Parser;
using DotnetSpider.DataFlow.Storage;
using DotnetSpider.Http;
using DotnetSpider.RequestSupplier;
using DotnetSpider.Scheduler.Component;
using DotnetSpider.Selector;
using DotnetSpider.Statistics.Store;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;

using MySql.Data.MySqlClient;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RabbitMQ.Client;
using Serilog;
using StringContent = DotnetSpider.Http.StringContent;

namespace DotnetSpider.Lawyee.Src
{
	public class LawSpider : Spider
	{
		public static bool isEmpty = true;
		#region 静态属性及公用方法
		public static bool isSchedulerEmpty = false;

		/// <summary>
		/// 是否启用Redis队列去重及错误的记录
		/// </summary>
		public static bool StartRedisQueue = true;

		/// <summary>
		/// 默认开启从Redis获取Token
		/// </summary>
		public static bool RedisToken = true;

		/// <summary>
		/// 浏览器UserAgent
		/// </summary>
		public static string UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";

		/// <summary>
		/// 解密API
		/// </summary>
		//public static string ApiUrl = "http://120.27.216.36:8080/website/getParam";
		public static string ApiUrl = "http://192.168.3.202:5000/website/parse/token";

		/// <summary>
		/// 查询的最大记录条数
		/// </summary>
		public static int MaxRecord = 1000;

		/// <summary>
		/// 单页面最大记录条数
		/// </summary>
		public static int PageSize = 1000;

		/// <summary>
		/// 查询过虑条件
		/// </summary>
		public static List<string> ConditionList = new List<string> {
			"s33,s39,s40",  //地域及法院 
			"s42"           //总数
		};

		/// <summary>
		/// 姓氏基础列表
		/// </summary>
		public static List<string> FamilyList = new List<string> {
			"王","李","张","刘","陈","杨","黄","孙","周","吴","徐","赵","朱","马","胡","郭","林","何","高","梁","郑","罗","宋","谢","唐","韩","曹","许","邓","萧","冯","曾","程","蔡","彭","潘","袁","于","董","余",
			"苏","叶","吕","魏","蒋","田","杜","丁","沈","姜","范","江","傅","钟","卢","汪","戴","崔","任","陆","廖","姚","方","金","邱","夏","谭","韦","贾","邹","石","熊","孟","秦","阎","薛","侯","雷","白","龙",
			"段","郝","孔","邵","史","毛","常","万","顾","赖","武","康","贺","严","尹","钱","施","牛","洪","龚","汤","陶","黎","温","莫","易","樊","乔","文","安","殷","颜","庄","章","鲁","倪","庞","邢","俞","翟",
			"蓝","聂","齐","向","申","葛","柴","伍","覃","骆","关","焦","柳","欧","祝","纪","尚","毕","耿","芦","左","季","管","符","辛","苗","詹","曲","欧","靳","祁","路","涂","兰","甘","裴","梅","童","翁","霍",
			"游","阮","尤","岳","柯","牟","滕","谷","舒","卜","成","饶","宁","凌","盛","查","单","冉","鲍","华","包","屈","房","喻","解","蒲","卫","简","时","连","车","项","闵","邬","吉","党","阳","司","费","蒙",
			"席","晏","隋","古","强","穆","姬","宫","景","米","麦","谈","柏","瞿","艾","沙","鄢","桂","窦","郁","缪","畅","巩","卓","褚","栾","戚","全","娄","甄","郎","池","丛","边","岑","农","苟","迟","保","商",
			"臧","佘","卞","虞","刁","冷","应","匡","栗","仇","练","楚","揭","师","官","佟","封","燕","桑","巫","敖","原","植","邝","仲","荆","储","宗","楼","干","苑","寇","盖","南","屠","鞠","荣","井","乐","银",
			"奚","明","麻","雍","花","闻","冼","木","郜","廉","衣","蔺","和","冀","占","公","门","帅","利","满", //百家氏
			"古","丽","娜","依","如","莎","奇","纳","奎","吐","迪","夏","努","露","祖","拉","尤","穆","默","德","不","音","沙","斯","苏","来","玉","素","达","吾","提","瓦","罕","力","亚","帕","萨","仙","肉","索",
			"伊","玛","尼","伯","孜","塔","里","克","板","尔","牙","库","甫","艾","合","坦","木","坎","吉","热","比","安","哈","蒂","曼",  //新疆西藏姓氏
			"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z" , //外国人名
			"广东","河南","内蒙古自治","黑龙江","新疆维吾尔自治","湖北","辽宁","山东","陕西","上海","贵州","重庆","西藏自治","安徽","福建","湖南","海南","江苏","青海","广西壮族自治","宁夏回族自治","江西","浙江",
			"河北","香港特别行政","山西","台湾","澳门特别行政","甘肃","四川","云南","北京","天津","吉林","汕头","佛山","珠海","江门","湛江","惠州","肇庆","汕尾","茂名","深圳","阳江","潮州","韶关","梅州","河源",
			"清远","东莞","云浮","揭阳","广州","中山","东沙群岛","濮阳","洛阳","三门峡","郑州","许昌","南阳","漯河","济源","平顶山","信阳","焦作","安阳","驻马店","商丘","新乡","鹤壁","周口","开封","乌海","巴彦淖尔",
			"呼伦贝尔","鄂尔多斯","包头","呼和浩特","通辽","赤峰","阿拉善盟","兴安盟","锡林郭勒盟","乌兰察布","大兴安岭地","七台河","大庆","鹤岗","伊春","绥化","佳木斯","齐齐哈尔","鸡西","双鸭山","哈尔滨","黑河",
			"牡丹江","博尔塔拉蒙古自治州","和田地","塔城地","阿勒泰地","克拉玛依","昌吉回族自治州","巴音郭楞蒙古自治州","喀什地","伊犁哈萨克自治州","乌鲁木齐","阿克苏地","克孜勒苏柯尔克孜自治州","哈密","吐鲁番","十堰",
			"宜昌","孝感","武汉","恩施土家族苗族自治州","黄冈","襄阳","荆门","随州","荆州","咸宁","黄石","鄂州","大连","葫芦岛","锦州","丹东","抚顺","沈阳","鞍山","铁岭","辽阳","盘锦","营口","朝阳","阜新","本溪","威海",
			"烟台","滨州","临沂","莱芜","淄博","青岛","聊城","德州","日照","菏泽","潍坊","济南","泰安","东营","济宁","枣庄","商洛","西安","汉中","安康","榆林","铜川","咸阳","宝鸡","延安","渭南","上海城","铜仁","六盘水",
			"黔东南苗族侗族自治州","遵义","黔南布依族苗族自治州","黔西南布依族苗族自治州","安顺","毕节","贵阳","重庆城","重庆郊","昌都","拉萨","那曲","日喀则","山南","林芝","阿里地","阜阳","淮北","铜陵","蚌埠","马鞍山",
			"池州","亳州","滁州","安庆","黄山","宣城","芜湖","六安","淮南","合肥","宿州","宁德","福州","龙岩","莆田","泉州","三明","厦门","漳州","南平","岳阳","衡阳","娄底","湘潭","益阳","长沙","常德","怀化","邵阳","张家界",
			"湘西土家族苗族自治州","株洲","永州","郴州","三沙","三亚","儋州","海口","宿迁","连云港","扬州","南京","南通","无锡","镇江","淮安","泰州","徐州","苏州","常州","盐城","海东","海南藏族自治州","海西蒙古族藏族自治州",
			"玉树藏族自治州","黄南藏族自治州","果洛藏族自治州","海北藏族自治州","西宁","钦州","桂林","百色","北海","贵港","河池","柳州","南宁","来宾","崇左","贺州","防城港","梧州","玉林","固原","中卫","银川","石嘴山","吴忠",
			"景德镇","九江","抚州","上饶","新余","赣州","鹰潭","南昌","吉安","萍乡","宜春","宁波","台州","嘉兴","舟山","温州","衢州","金华","丽水","杭州","绍兴","湖州","邢台","邯郸","唐山","承德","张家口","廊坊","沧州","衡水",
			"秦皇岛","保定","石家庄","阳泉","太原","临汾","大同","晋城","忻州","长治","运城","朔州","晋中","吕梁","兰州","金昌","嘉峪关","酒泉","平凉","白银","张掖","甘南藏族自治州","临夏回族自治州","陇南","天水","武威","定西",
			"庆阳","广元","南充","达州","眉山","德阳","遂宁","巴中","广安","资阳","成都","绵阳","内江","宜宾","自贡","雅安","攀枝花","乐山","甘孜藏族自治州","泸州","阿坝藏族羌族自治州","凉山彝族自治州","昭通","曲靖",
			"红河哈尼族彝族自治州","丽江","西双版纳傣族自治州","保山","文山壮族苗族自治州","大理白族自治州","怒江傈僳族自治州","迪庆藏族自治州","玉溪","普洱","昆明","楚雄彝族自治州","德宏傣族景颇族自治州","临沧","北京城",
			"天津城","长春","辽源","吉林","白城","松原","四平","延边朝鲜族自治州","白山","通化",


		};

		public static List<FYEntity> fylist = new List<FYEntity>();
	
		/// <summary>
		/// 配置文件对象
		/// </summary>
		public static SpiderOptions Options_;

		/// <summary>
		/// 生成随机数
		/// </summary>
		/// <param name="min"></param>
		/// <param name="max"></param>
		/// <returns></returns>
		public static int GetRandom(int min, int max)
		{
			Random random = new Random(GetRandomSeed());
			return random.Next(min, max);
		}

		/// <summary>
		/// 随机种子
		/// </summary>
		/// <returns></returns>
		public static int GetRandomSeed()
		{
			byte[] data = new byte[4];
			new System.Security.Cryptography.RNGCryptoServiceProvider().GetBytes(data);
			return BitConverter.ToInt32(data, 0);
		}

		/// <summary>
		/// 生成指定数位随机数
		/// </summary>
		/// <param name="num"></param>
		/// <returns></returns>
		public static string RandomNumber(int num)
		{
			char[] randomSeed = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".ToCharArray();

			List<string> listRandom = new List<string>();

			for (int i = 0; i < num; i++)
			{
				int index = GetRandom(0, randomSeed.Length);
				listRandom.Add(randomSeed[index].ToString());
			}

			return string.Join("", listRandom);

		}


		/// <summary>
		/// 32位MD5加密
		/// </summary>
		/// <param name="password"></param>
		/// <returns></returns>
		public static string MD5Encrypt32(string password)
		{
			string cl = password;
			string pwd = "";
			MD5 md5 = MD5.Create(); //实例化一个md5对像
									// 加密后是一个字节类型的数组，这里要注意编码UTF8/Unicode等的选择　
			byte[] s = md5.ComputeHash(Encoding.UTF8.GetBytes(cl));
			// 通过使用循环，将字节类型的数组转换为字符串，此字符串是常规字符格式化所得
			for (int i = 0; i < s.Length; i++)
			{
				// 将得到的字符串使用十六进制类型格式。格式后的字符是小写的字母，如果使用大写（X）则格式后的字符是大写字符 
				pwd = pwd + s[i].ToString("X");
			}
			return pwd;
		}


		


		public static async Task RunAsync()
		{
			//var builder = Builder.CreateDefaultBuilder<LawSpider>();  
			var builder = Builder.CreateDefaultBuilder<LawSpider>(options =>
			{
				// 每秒 1 个请求
				options.Speed = 20;

				////最大深度
				//options.Depth = 1;

				//最大的队列并发数
				options.RequestedQueueCount = 1000 * 10 * 10 * 10 * 1000;

				// 请求超时
				options.RequestTimeout = 10 * 24 * 60 * 60;

				//尝试次数
				options.RetriedTimes = 50000;

				//Scheduler持续时间
				options.EmptySleepTime = 60 * 60 * 24 * 1500;

				//爬虫是否使用代理
				//options.UseProxy = true;

				//去除外链
				//options.RemoveOutboundLinks = true; 

			});

			builder.UseSerilog();
			builder.UseQueueDistinctBfsScheduler<HashSetDuplicateRemover>();
			await builder.Build().RunAsync();
		}

		public LawSpider(
			IOptions<SpiderOptions> options,
			SpiderServices services,
			ILogger<Spider> logger , IStatisticsStore _statisticsStore) : base(options, services, logger, _statisticsStore)
		{
			Options_ = options.Value;
		}

		#region 未处理的重载

		public override void Dispose()
		{
			base.Dispose();
		}

		public override bool Equals(object obj)
		{
			return base.Equals(obj);
		}

		public override int GetHashCode()
		{
			return base.GetHashCode();
		}

		public override Task StartAsync(CancellationToken cancellationToken)
		{
			return base.StartAsync(cancellationToken);
		}

		public override Task StopAsync(CancellationToken cancellationToken)
		{
			return base.StopAsync(cancellationToken);
		}

		public override string ToString()
		{
			return base.ToString();
		}

		protected override Spider AddDataFlow(IDataFlow dataFlow)
		{
			return base.AddDataFlow(dataFlow);
		}

		protected override void AddRequestSupplier(IRequestSupplier requestSupplier)
		{
			base.AddRequestSupplier(requestSupplier);
		}

		protected override void ConfigureRequest(Request request)
		{
			base.ConfigureRequest(request);
		}

		protected override Task ExecuteAsync(CancellationToken stoppingToken)
		{
			return base.ExecuteAsync(stoppingToken);
		}


		protected override (string Id, string Name) GetIdAndName()
		{
			//return base.GetIdAndName();
			return (Guid.NewGuid().ToString(), "中国裁判文书网");
		}


		#endregion

		#endregion

		protected override async Task InitializeAsync(CancellationToken stoppingToken)
		{
			//增加法院信息
			CommonHelper.AddFYjson();

			//文书过虑处理类
			AddDataFlow(new cprqDataParser());
			AddDataFlow(new s2DataParser());
			AddDataFlow(new s17DataParser());


			//////文书查询处理类
			AddDataFlow(new QueryDocParser());

			//////文书详情处理类
			AddDataFlow(new DocInfoSearch());

			//AddDataFlow(new MySqlStorage());      //Mysql存储类
			AddDataFlow(new SqlServerStorage());   //SQL Server存储类

			OnSchedulerEmpty += LawSpider_OnSchedulerEmpty;

			OnError += updateToken;

			new TimedBackgroundService().StartAsync();

			//string runtimes = ConfigurationManager.AppSettings["runtimes"];
			//if (runtimes != "2")
			//{
			//	using (var conn = new SqlConnection(ConfigurationManager.AppSettings["StorageConnectionStringForDate"]))
			//	{
			//		string MachId = ConfigurationManager.AppSettings["MachId"];
			//		conn.Execute($"update dateOne  WITH (TABLOCKX) set started = 0 where MachId = @MachId AND Pcount < 2 AND dateOne.dateOne > (SELECT TOP 1 dateOne FROM dateOne where totalcount != 0 and totalcount = completed and MachId = @MachId ORDER BY dateOne DESC)", new { MachId = MachId });
			//	}
			//}

			StartAuto();

		}

		private async void updateToken(Request arg1, Response arg2)
		{
			string cookie = string.Empty;
			string url = string.Empty;
			try
			{
				TokenEntity tokenEntity = new TokenEntity();

				Tuple<string, string> tokenItem = tokenEntity.DonetTokenEx();

				cookie = tokenItem.Item1;
				url = tokenItem.Item2;

			}
			catch (Exception ex)
			{
				Log.Logger.Error(ex.ToString());

				throw ex;
			}

			Request request = arg1.Clone();
			request.RequestUri = new Uri(url);
			request.Cookie = cookie;
			await AddRequestsAsync(request);
		}

		private void LawSpider_OnSchedulerEmpty()
		{
			try
			{
				if (isSchedulerEmpty)
				{
					StartAuto(1);
					isSchedulerEmpty = false;
				}
				else
				{
					Console.WriteLine("请求为0了，，，，，，，，等待加入请求！！！！！！！！！！！！！！！！！！！！！！！！！！！");

				}
			}
			catch (Exception ex)
			{
				Log.Logger.Error(ex.Message+","+ex.StackTrace);
				Console.WriteLine(ex.Message + "," + ex.StackTrace);
			}
		}


		



		public void StartAuto(int count =0)
		{
			try
			{
				using (var conn = new SqlConnection(ConfigurationManager.AppSettings["StorageConnectionStringForDate"]))
				{
					string MachId = ConfigurationManager.AppSettings["MachId"];
					string StartDate = ConfigurationManager.AppSettings["StartDate"]; 
					string EndDate = ConfigurationManager.AppSettings["EndDate"];
					//var result = conn.ExecuteReader($"select dateOne from dateOne where started = 0  and (totalcount = 0 or totalcount > completed) and (MachId = @MachId ) and dateOne >= @StartDate and dateOne <= @EndDate ORDER BY dateOne asc", new { MachId = MachId, StartDate = StartDate, EndDate = EndDate });
					//DataTable dt = CommonHelper.DataTableToIDataReader(result);
					//for (int i = 0; i < dt.Rows.Count; i++)
					//{
					//	StartSingle(dt.Rows[i]["dateOne"].ToString(), 0);
					//}
					var result = conn.ExecuteScalar($"select top 1 dateOne from dateOne where started = 0  and (totalcount = 0 or totalcount > completed) and (MachId = @MachId ) and dateOne >= @StartDate and dateOne <= @EndDate ORDER BY dateOne asc", new { MachId = MachId, StartDate = StartDate, EndDate = EndDate });
					if (result != null)
					{ 
						Console.WriteLine(string.Format("{0} DoubleTime   .................................................................", result.ToString()));
						StartSingle(result.ToString(), count: count);
					}
					conn.Execute($"update dateOne  WITH (TABLOCKX) set started = 1 , Pcount = Pcount+1 where dateone = @dateone", new { dateone = result });
				}
			}
			catch (Exception ex)
			{
				Console.WriteLine(string.Format("DoubleTime   .................................................................{0}", ex.Message));
				//throw ex;
			}
		}


		/// <summary>
		/// 
		/// </summary>
		/// <param name="startDate"></param>
		/// <param name="totalDays">0代表当天</param>
		/// <returns></returns>
		public async void StartSingle(string startDate , int totalDays = 0,int count =0)
		{
			try
			{
				List<Request> listRequest = new List<Request>();

				Request request = cprqDataParser.ConditionRequst(startDate, totalDays);

				listRequest.Add(request);

				//context.AddFollowRequests(request);

				await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray(), count);

				//await _services.StatisticsClient.IncreaseTotalAsync(Id, _services.Scheduler.Total);

			}
			catch (Exception ex)
			{

				throw ex;
			}
			
		}
		
		/// <summary>
		/// 法院名称list
		/// </summary>
		

		#region 查询左边菜单 request
		/// <summary>
		/// 查询左边菜单
		/// </summary>
		/// <param name="queryCondition"></param>
		/// <param name="pageId"></param>
		/// <param name="context"></param>
		/// <param name="ua"></param>
		/// <returns></returns>
		public static Request CreateDocDataItemRequest(string queryCondition,string ValidatorName, string pageId = "", DataContext context = null, string ua = "")
		{
			Request request = null;
			string cookie = string.Empty;
			string url = string.Empty;

			try
			{
				TokenEntity tokenEntity = new TokenEntity();

				Tuple<string, string> tokenItem = tokenEntity.DonetTokenEx();

				cookie = tokenItem.Item1;
				url = tokenItem.Item2;

			}
			catch (Exception ex)
			{
				Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);
				throw ex;
				//return request;
			}

			List<string> listParam = new List<string>();

			if (string.IsNullOrEmpty(pageId))
			{
				pageId = System.Guid.NewGuid().ToString().Replace("-", "");
			}

			listParam.Add("pageId" + "=" + pageId);
			listParam.Add("groupFields" + "=" +
				HttpUtility.UrlEncode(string.Join(";", ConditionList.ToArray()), Encoding.UTF8).Replace("+", "%20"));
			listParam.Add("queryCondition" + "=" +
				HttpUtility.UrlEncode(queryCondition, Encoding.UTF8).Replace("+", "%20"));
			listParam.Add("cfg" + "=" +
				HttpUtility.UrlEncode("com.lawyee.judge.dc.parse.dto.SearchDataDsoDTO@leftDataItem", Encoding.UTF8).Replace("+", "%20"));
			listParam.Add("__RequestVerificationToken" + "=" + RandomNumber(24));

			if (context == null)
			{
				request = new Request(url);
			}
			else
			{

				request = context.CreateNewRequest(url);
			}

			request.Name = ValidatorName;

			request.SetProperty("pageId", pageId);

			return QueryDocRequest(ref request, cookie, string.Join("&", listParam.ToArray()));

		}
		#endregion

		/// <summary>
		/// 翻页处理
		/// </summary>
		/// <param name="count"></param>
		/// <param name="pageId"></param>
		/// <param name="condition"></param>
		/// <param name="context"></param>
		public static void ScrollPage(int count, string pageId, JArray condition, DataContext context)
		{
			//int pageNum = (int)(count / PageSize) + 1;   ///   (int)5/6 = ?
			int pageNum =  1;   ///   一页返回完

			Request request = null;

			for (int i = 0; i < pageNum; i++)
			{
				request = CreateQueryDocRequest(condition, i + 1, pageId, context);

				if (request != null)
				{
					context.AddFollowRequests(request);
				}
				else
				{
					Log.Logger.Warning(string.Format("page:{0}\t", i + 1) + JsonConvert.SerializeObject(condition));
				}

			}
		}


		/// <summary>
		/// 文档搜索请求构造类
		/// </summary>
		/// <param name="url"></param>
		/// <param name="cookie"></param>
		/// <param name="ua"></param>
		/// <returns></returns>
		public static Request CreateQueryDocRequest(JArray condition, int pageNum = 1, string pageId = "", DataContext context = null, string ua = "")
		{

			Request request = null;

			List<string> listParam = new List<string>();
			listParam.Add("sortFields" + "=" +
				HttpUtility.UrlEncode("s50:desc", Encoding.UTF8).Replace("+", "%20"));
			listParam.Add("ciphertext" + "=" + "");
			listParam.Add("pageNum" + "=" + pageNum.ToString());
			listParam.Add("pageSize" + "=" + PageSize.ToString());
			listParam.Add("facetLimit" + "=" + PageSize.ToString());
			listParam.Add("queryCondition" + "=" +
				HttpUtility.UrlEncode(JsonConvert.SerializeObject(condition), Encoding.UTF8).Replace("+", "%20"));
			listParam.Add("cfg" + "=" +
				HttpUtility.UrlEncode("com.lawyee.judge.dc.parse.dto.SearchDataDsoDTO@queryDoc", Encoding.UTF8).Replace("+", "%20"));

			string param = string.Join("&", listParam.ToArray());

			//查询条件的md5
			string md5 = MD5Encrypt32(param);

			try
			{


				if (string.IsNullOrEmpty(pageId))
				{
					pageId = System.Guid.NewGuid().ToString().Replace("-", "");
				}

				listParam.Add("pageId" + "=" + pageId);

				string cookie = string.Empty;
				string url = string.Empty;

				try
				{
					TokenEntity tokenEntity = new TokenEntity();

					Tuple<string, string> tokenItem = tokenEntity.DonetTokenEx();

					cookie = tokenItem.Item1;
					url = tokenItem.Item2;

				}
				catch (Exception ex)
				{
					Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);
					//return request;
					throw ex;
				}


				if (context == null)
				{
					request = new Request(url);

				}
				else
				{
					request = context.CreateNewRequest(url);
				}

				request.Name = "queryDoc";

				request.SetProperty("md5", md5);

				listParam.Add("__RequestVerificationToken" + "=" + RandomNumber(24));

				request = QueryDocRequest(ref request, cookie, string.Join("&", listParam.ToArray()));

				
			}
			catch (Exception ex)
			{
				Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);
				throw ex;
			}
			return request;

		}

		/// <summary>
		/// 文书基础请求构造类
		/// </summary>
		/// <param name="request"></param>
		/// <param name="cookie"></param>
		/// <param name="listParam"></param>
		/// <param name="ua"></param>
		/// <returns></returns>
		public static Request QueryDocRequest(ref Request request, string cookie, string param, string ua = "")
		{

			try
			{
				request.UserAgent = string.IsNullOrEmpty(ua) ? UserAgent : ua;
				request.SetHeader("accept-encoding", "gzip, deflate, br");
				request.SetHeader("accept-language", "zh-CN,zh;q=0.8");
				request.SetHeader("Origin", "http://wenshu.court.gov.cn");
				request.SetHeader("x-requested-with", "XMLHttpRequest");
				request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
				request.Referer = "http://wenshu.court.gov.cn/";
				request.Cookie = cookie;
				request.Method = "POST";
				RequestContent content = new StringContent(param, "application/x-www-form-urlencoded");
				request.SetContent(content);

			}
			catch (Exception ex)
			{
				Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);
				throw ex;
			}
			return request;

		}

		


		/// <summary>
		/// 调用Redis处理异常类
		/// </summary>
		/// <param name="listRequest"></param>
		public static void AttarchRedisRequest(List<Request> listRequest)
		{
			Request request = new Request("https://www.baidu.com");

			request.Name = "RedisParser";

			listRequest.Add(request);
		}

		#region ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd

		/// <summary>queryDocRequest
		/// 文书查询结构解析类
		/// </summary>
		public class QueryDocParser : DataParser
		{

			public QueryDocParser()
			{
				// AddRequiredValidator("news\\.cnblogs\\.com/n/page"); 
				AddRequiredValidatorEx("queryDoc");
			}

			/// <summary>
			/// 文书查询页面解析，获取DocId，生成详情页请求
			/// </summary>
			/// <param name="context"></param>
			/// <returns></returns>
			protected override Task Parse(DataContext context)
			{
				try
				{

					var content = CommonHelper.CommonParser(context);

					if (string.IsNullOrEmpty(content) || !content.StartsWith("{") || !content.EndsWith("}"))
					{
						CommonHelper.RetryRequest(context);
						return Task.CompletedTask;
					}


					JObject jdoc = JObject.Parse(content);

					string resultCount = jdoc["queryResult"]["resultCount"].ToString();

					JArray resultList = (JArray)jdoc["queryResult"]["resultList"];

					JArray rowKeyList = new JArray();

					bool first = true;

					int tempcount = 0;

					foreach (JObject resultItem in resultList)
					{
						tempcount++;
						try
						{

							resultItem.Add(new JProperty("queryParams", jdoc["queryParams"]));


							string rowkey = resultItem["rowkey"].ToString();

							string docIdStatus = string.Empty;
							bool doubletime = false;

							rowKeyList.Add(rowkey);
							int currentcount = 0;

							using (var conn = new SqlConnection(Options_.StorageConnectionString))
							{
								try
								{
									var result = conn.ExecuteScalar($"select 1 from record where docId = @docId", new { docId = rowkey });
									//Thread.Sleep(1000);
									if (result != null && result.ToString() == "1")
									{
										Console.WriteLine(resultItem["31"].ToString() +"重复:" + rowkey);
										doubletime = true;
									}
								}
								catch (Exception ex)
								{
									Console.WriteLine(ex.Message + "," + ex.StackTrace);
									doubletime = true;
									throw ex;
								}
							}

							if (doubletime)
							{
								continue;

							}

							string cookie = string.Empty;
							string url = string.Empty;

							TokenEntity tokenEntity = new TokenEntity();

							Tuple<string, string> tokenItem = tokenEntity.DonetTokenEx();

							cookie = tokenItem.Item1;
							url = tokenItem.Item2;

							var request = CreateSearchDocRequest(context, url, cookie, rowkey);

							request.SetProperty("DocId", resultItem.ToString());

							context.AddFollowRequests(request);

						}
						catch (Exception ex)
						{
							CommonHelper.RetryRequest(context);
							Console.WriteLine(ex.Message + "," + ex.StackTrace);
							//throw ex;
						}

					}


				}
				catch (Exception ex)
				{
					CommonHelper.RetryRequest(context);
					Console.WriteLine(ex.Message + "," + ex.StackTrace);
					Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);
					throw ex;
				}

				return Task.CompletedTask;
			}



			/// <summary>
			/// 文书文档内容详情请求构造类
			/// </summary>
			/// <param name="context"></param>
			/// <param name="url"></param>
			/// <param name="cookie"></param>
			/// <param name="docId"></param>
			/// <param name="ua"></param>
			/// <returns></returns>
			public static Request CreateSearchDocRequest(DataContext context, string url, string cookie, string docId, string ua = "")
			{
				List<string> listParam = new List<string>();
				string pageId = System.Guid.NewGuid().ToString().Replace("-", "");
				listParam.Add("docId" + "=" + docId);
				listParam.Add("sortFields" + "=" + "s50:desc");
				listParam.Add("ciphertext" + "=" + "");
				listParam.Add("cfg" + "=" +
					HttpUtility.UrlEncode("com.lawyee.judge.dc.parse.dto.SearchDataDsoDTO@docInfoSearch",
					Encoding.UTF8).Replace("+", "%20"));

				listParam.Add("__RequestVerificationToken" + "=" + RandomNumber(24));

				Request request = context.CreateNewRequest(url);

				request.Name = "docInfoSearch";

				return QueryDocRequest(ref request, cookie, string.Join("&", listParam.ToArray()));
			}

		}

		/// <summary>
		/// 文书文档内容解析类
		/// </summary>  
		public class DocInfoSearch : DataParser
		{

			public DocInfoSearch()
			{

				AddRequiredValidatorEx("docInfoSearch");
			}


			/// <summary>
			/// 文书文档详情页解析
			/// </summary>
			/// <param name="context"></param>
			/// <returns></returns>
			protected override Task Parse(DataContext context)
			{

				try
				{

					var content = CommonHelper.CommonParser(context);

					if (string.IsNullOrEmpty(content) || !content.StartsWith("{") || !content.EndsWith("}"))
					{
						CommonHelper.RetryRequest(context);
						return Task.CompletedTask;
					}

					JObject record = JObject.Parse(context.Request.Properties["DocId"].ToString());

					string docId = record["rowkey"].ToString();

					record.Add(new JProperty("doc", JObject.Parse(content)));

					string current = record["doc"]["s31"].ToString();

					string typeName = typeof(Record).FullName;

					context.AddData(typeName,
						new Record
						{
							DocId = docId,
							Content = JsonConvert.SerializeObject(record),
							Current = current

						});
				}
				catch (Exception ex)
				{
					Log.Logger.Error(ex.StackTrace +"!!!"+ context.Response.ReadAsString().TrimStart());
					CommonHelper.RetryRequest(context);
				}
				return Task.CompletedTask;
			}
		}

		/// <summary>
		/// 文书存储字段类
		/// </summary>
		public class Record
		{
			public string DocId { get; set; }

			public string Content { get; set; }

			public string Current { get; set; }

		}
		/// <summary>
		/// SQL Server储存类
		/// </summary>
		public class SqlServerStorage : StorageBase
		{

			public override async Task InitAsync()
			{
			}


			protected override async Task StoreAsync(DataContext context)
			{
				var typeName = typeof(Record).FullName;
				var data = (Record)context.GetData(typeName);
				try
				{
					if (data != null)
					{
						using (var conn = new SqlConnection(Options_.StorageConnectionString))
						{
							var result = conn.ExecuteScalar($"select 1 from record where docId = @docId", new { docId = data.DocId });
							if (result == null || result.ToString() != "1")
							{
								conn.Execute(
							$"INSERT INTO record ([docId], [content],[current]) VALUES (@DocId,@Content,@Current) ",
							data);
							}
						}
					}
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(ConfigurationManager.AppSettings["StorageConnectionStringForDate"]))
					{
						if (data!=null)
						{
							conn.Execute(
						$"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml( ex.Message+ex.StackTrace) + ",data:" + data.DocId + "','" + DateTime.Now.ToString() + "') ");
						}
						else
						{
							conn.Execute(
						$"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message+ex.StackTrace )+ ",  :SqlServerStorage,StoreAsync' ,'" + DateTime.Now.ToString() + "') " );
						}
					}
					CommonHelper.RetryRequest(context);
					throw ex;
				}
			}
		}


		/// <summary>
		/// Redis中遍历异常队列数据处理
		/// </summary>
		public class RedisParser : DataParser
		{
			public RedisParser()
			{
				AddRequiredValidatorEx("RedisParser");
			}
			protected override Task Parse(DataContext context)
			{
				return Task.CompletedTask;
			}
		}

		/// <summary>
		/// 获取Token类
		/// </summary>
		public class TokenEntity
		{

			/// <summary>
			/// 获取Token和cookie
			/// </summary>
			///// <returns></returns>
			//public Tuple<string, string> DonetToken()
			//{
			//	Tuple<string, string> tokenItem = null;

			//	try
			//	{
			//		string url = ApiUrl;

			//		ResultModel resultModel = HttpGet(url);

			//		if (!resultModel.Pass)
			//		{
			//			Log.Logger.Error(resultModel.Message);
			//		}
			//		else
			//		{

			//			if (resultModel.html.StartsWith("{") && resultModel.html.EndsWith("}"))
			//			{

			//				JObject jobj_token = JObject.Parse(resultModel.html);

			//				if (jobj_token.Property("cookie") != null &&
			//					jobj_token.Property("hifJzoc9") != null)
			//				{

			//					string cookie = jobj_token["cookie"].ToString();

			//					string token = jobj_token["hifJzoc9"].ToString();

			//					tokenItem = new Tuple<string, string>(cookie, token);

			//				}

			//			}

			//		}
			//	}
			//	catch (Exception ex)
			//	{
			//		Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);

			//		throw ex;
			//	}

			//	return tokenItem;
			//}

			public Tuple<string, string> DonetTokenEx()
			{
				return new Tuple<string, string>("a=b", "http://wenshu.court.gov.cn/website/parse/rest.q4w?" + Guid.NewGuid().ToString());

				
				//int retry_index = 0;
				//int retry_max = 1000;
				//string cookie = string.Empty;
				//string url = string.Empty;
				//string urlStarts = "http://wenshu.court.gov.cn/website/parse/rest.q4w?HifJzoc9=";

				//try
				//{
				//	do
				//	{

				//		if (RedisToken)
				//		{

				//			try
				//			{
				//				string tokenText = RedisHelpers.ListRightPop<string>("LawyeeApi");

				//				if (!string.IsNullOrEmpty(tokenText) && tokenText.StartsWith("{") && tokenText.EndsWith("}"))
				//				{
				//					JObject jobj = JObject.Parse(tokenText);

				//					if (jobj.Property("cookie") != null && jobj.Property("hifJzoc9") != null)
				//					{
				//						cookie = jobj["cookie"].ToString();
				//						string DonetTokenEx = ConfigurationManager.AppSettings["DonetTokenEx"];
				//						if (string.IsNullOrEmpty(DonetTokenEx) || DonetTokenEx == "false")
				//						{
				//							url = "http://wenshu.court.gov.cn/website/parse/rest.q4w";

				//						}
				//						else
				//						{
				//							url = urlStarts + jobj["hifJzoc9"].ToString();

				//						}
				//					}

				//				}

				//				if (string.IsNullOrEmpty(cookie))
				//				{
				//					Log.Logger.Warning("服务器获取Token失败，等待重新获取中......");
				//					Thread.Sleep(1000);
				//				}

				//			}
				//			catch (Exception ex)
				//			{

				//				Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);
				//				//throw ex;
				//			}


				//		}
				//		else
				//		{
				//			//获取加密的token和cookie
				//			Tuple<string, string> tokenItem = DonetToken();

				//			if (tokenItem == null ||
				//				string.IsNullOrEmpty(tokenItem.Item1))
				//			{

				//				Log.Logger.Warning("服务器获取Token失败，等待重新获取中......");
				//				Thread.Sleep(1000);
				//			}
				//			else
				//			{
				//				cookie = tokenItem.Item1;
				//				url = urlStarts + tokenItem.Item2;

				//			}
				//		}

				//	} while (retry_index < retry_max && string.IsNullOrEmpty(cookie));

				//	Tuple<string, string> tokenResult = new Tuple<string, string>(cookie, url);

				//	return tokenResult;

				//}
				//catch (Exception ex)
				//{
				//	Log.Logger.Error(ex.Message + "!!!"+ ex.StackTrace);
				//	throw ex;
				//}
			}

			/// <summary>
			/// 请求服务器获取加密token
			/// </summary>
			/// <param name="url">服务器地址</param>
			/// <returns></returns>
			//public ResultModel HttpGet(string url)
			//{
			//	ResultModel resultModel = new ResultModel();

			//	try
			//	{

			//		System.Net.WebHeaderCollection header = new System.Net.WebHeaderCollection();
			//		header.Add("accept-encoding: gzip, deflate, br");
			//		header.Add("accept-language: zh-CN,zh;q=0.8");
			//		HttpHelper httpHelper = new HttpHelper();
			//		HttpItem httpItem = new HttpItem()
			//		{
			//			URL = url,
			//			Method = "GET",
			//			Header = header,
			//			Accept = "*/*",
			//			Timeout = 30 * 1000,
			//			UserAgent = UserAgent,
			//			ContentType = null

			//		};

			//		HttpResult httpResult = httpHelper.GetHtml(httpItem);

			//		if (httpResult.StatusCode == HttpStatusCode.OK)
			//		{
			//			resultModel.Pass = true;

			//			resultModel.html = httpResult.Html;

			//		}
			//		else
			//		{
			//			if (httpResult.Header != null)
			//			{
			//				resultModel.html = httpResult.Header.ToString();

			//			}
			//			else
			//			{

			//				resultModel.html = "服务器无响应！";

			//			}

			//		}
			//	}
			//	catch (Exception ex)
			//	{

			//		resultModel.Message = ex.Message + "!!!"+ ex.StackTrace;
			//		throw ex;
			//	}

			//	return resultModel;

			//}


			//#region 备用查询方法

			///// <summary>
			///// 查询过滤条件
			///// </summary>
			///// <param name="url"></param>
			///// <param name="queryCondition"></param>
			///// <param name="cookie"></param>
			///// <returns></returns>
			//public ResultModel HttpCondition(string url, string queryCondition, string cookie, string pageId = "")
			//{
			//	ResultModel resultModel = new ResultModel();

			//	try
			//	{

			//		List<string> listParam = new List<string>();
			//		pageId = string.IsNullOrEmpty(pageId) ? System.Guid.NewGuid().ToString().Replace("-", "") : pageId;
			//		listParam.Add("pageId" + "=" + pageId);
			//		//listParam.Add("groupFields" + "=" + HttpUtility.UrlEncode("s45;s11;s4;s33;s42;s8;s6;s44", Encoding.UTF8).Replace("+", "%20")); 
			//		listParam.Add("groupFields" + "=" + HttpUtility.UrlEncode(string.Join(";", ConditionList.ToArray()), Encoding.UTF8).Replace("+", "%20"));
			//		listParam.Add("queryCondition" + "=" + HttpUtility.UrlEncode(queryCondition, Encoding.UTF8).Replace("+", "%20"));
			//		listParam.Add("cfg" + "=" + HttpUtility.UrlEncode("com.lawyee.judge.dc.parse.dto.SearchDataDsoDTO@leftDataItem", Encoding.UTF8).Replace("+", "%20"));
			//		listParam.Add("__RequestVerificationToken" + "=" + "8hP6xtxGFVHdMEhO5gq1p63O");


			//		System.Net.WebHeaderCollection header = new System.Net.WebHeaderCollection();
			//		header.Add("accept-encoding: gzip, deflate, br");
			//		header.Add("accept-language: zh-CN,zh;q=0.8");
			//		header.Add("X-Requested-With: XMLHttpRequest");
			//		header.Add("Origin: http://wenshu.court.gov.cn");
			//		HttpHelper httpHelper = new HttpHelper();
			//		HttpItem httpItem = new HttpItem()
			//		{
			//			URL = url,
			//			Method = "POST",
			//			Header = header,
			//			Accept = "application/json, text/javascript, */*; q=0.01",
			//			Referer = "http://wenshu.court.gov.cn/",
			//			Timeout = 30 * 1000,
			//			UserAgent = UserAgent,
			//			ContentType = "application/x-www-form-urlencoded; charset=UTF-8",
			//			PostEncoding = Encoding.UTF8,
			//			Postdata = string.Join("&", listParam),
			//			Cookie = cookie

			//		};

			//		HttpResult httpResult = httpHelper.GetHtml(httpItem);

			//		if (httpResult.StatusCode == HttpStatusCode.OK)
			//		{
			//			resultModel.Pass = true;

			//			resultModel.html = httpResult.Html;

			//		}
			//		else
			//		{
			//			resultModel.html = httpResult.Header.ToString();

			//		}


			//	}
			//	catch (Exception ex)
			//	{

			//		resultModel.Message = ex.Message + "!!!"+ ex.StackTrace;
			//		throw ex;
			//	}
			//	return resultModel;
			//}


			//public ResultModel HttpConditionProcess(string queryCondition, string pageId = "")
			//{
			//	ResultModel resultModel = new ResultModel();

			//	try
			//	{

			//		TokenEntity tokenEntity = new TokenEntity();

			//		Tuple<string, string> tokenItem = tokenEntity.DonetToken();
			//		String cookie = tokenItem.Item1;
			//		string url = "http://wenshu.court.gov.cn/website/parse/rest.q4w?HifJzoc9=" + tokenItem.Item2;

			//		resultModel = HttpCondition(url, queryCondition, cookie, pageId);

			//	}
			//	catch (Exception ex)
			//	{
			//		resultModel.Message = ex.Message + "!!!"+ ex.StackTrace;
			//		throw ex;
			//	}
			//	return resultModel;
			//}
			//#endregion
		}
	
		public class FYjson
		{
			public string code { get; set; }
			public string id { get; set; }
			public string name { get; set; }
			public string parentid { get; set; }
			public List<FYjson2> child { get; set; }
		}
		public class FYjson2
		{
			public string code { get; set; }
			public string id { get; set; }
			public string name { get; set; }
			public string parentid { get; set; }
			public List<FYjson1> child { get; set; }
		}
		public class FYjson1
		{
			public string code { get; set; }
			public string id { get; set; }
			public string name { get; set; }
			public string parentid { get; set; }
			public List<FYEntity> child { get; set; }
		}
		public class FYEntity
		{
			public string code { get; set; }
			public string id { get; set; }
			public string name { get; set; }
			public string parentid { get; set; }
		}
		#endregion

	}



}
